{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "import math\n",
    "import time\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def conv_op(input_op , name , kh , kw , n_out , dh , dw , p):\n",
    "    n_in = input_op.get_shape()[-1].value\n",
    "    with tf.name_scope(name) as scope:\n",
    "        kernel = tf.get_variable(scope+'w', shape = [kh, kw, n_in , n_out], dtype = tf.float32, initializer= tf.contrib.layers.xavier_initializer_conv2d())\n",
    "        conv = tf.nn.conv2d(input_op , kernel , (1,dh,dw,1), padding= 'SAME')\n",
    "        bias_init_val = tf.constant(0.0, shape= [n_out] , dtype= tf.float32)\n",
    "        biases = tf.Variable(bias_init_val , trainable= True , name= 'b')\n",
    "        z = tf.nn.bias_add(conv , biases)\n",
    "        activation = tf.nn.relu(z, name= scope)\n",
    "        p += [kernel , biases]\n",
    "        return activation\n",
    "\n",
    "def fc_op(input_op, name , n_out , p):\n",
    "    n_in = input_op.get_shape()[-1].value\n",
    "    with tf.name_scope(name) as scope:\n",
    "        kernel = tf.get_variable(scope+'w' , shape= [n_in, n_out] , dtype= tf.float32 , initializer= tf.contrib.layers.xavier_initializer())\n",
    "        biases = tf.Variable(tf.constant(0.0 , shape= [        n_out], dtype= tf.float32), name= 'b')\n",
    "        activation = tf.nn.relu_layer(input_op , kernel , biases, name= scope)\n",
    "        p += [kernel , biases]\n",
    "        return  activation\n",
    "\n",
    "def mpool_op(input_op , name , kh , kw , dh , dw):\n",
    "    return tf.nn.max_pool(input_op, ksize= [1,kh,kw,1] , strides= [1,dh , dw,1], padding= 'SAME' , name= name)\n",
    "\n",
    "def inference_op_4(input_op , keep_prob):\n",
    "    p = []\n",
    "\n",
    "    conv1_1 = conv_op(input_op , name= 'conv1_1' , kh= 3, kw= 3, n_out= 64 , dh= 1, dw= 1 , p=p)\n",
    "    conv1_2 = conv_op(conv1_1 , name= 'conv1_2' , kh= 3, kw= 3, n_out= 64 , dh= 1, dw= 1 , p=p)\n",
    "\n",
    "    pool1 = mpool_op(conv1_2 , name= 'pool1' , kh= 2, kw= 2, dh= 2, dw=2)\n",
    "\n",
    "    conv2_1 = conv_op(pool1 , name='conv2_1', kh= 3, kw= 3, n_out= 128 , dh= 1, dw= 1 , p=p)\n",
    "    conv2_2 = conv_op(conv2_1 , name= 'conv2_2' , kh= 3, kw= 3, n_out= 128 , dh= 1, dw= 1 , p=p)\n",
    "\n",
    "    pool2 = mpool_op(conv2_2 , name= 'pool2' , kh= 2, kw= 2, dh= 2, dw=2)\n",
    "\n",
    "    conv3_1 = conv_op(pool2 , name='conv3_1', kh= 3, kw= 3, n_out= 256 , dh= 1, dw= 1 , p=p)\n",
    "    conv3_2 = conv_op(conv3_1 , name='conv3_2', kh= 3, kw= 3, n_out= 256 , dh= 1, dw= 1 , p=p)\n",
    "    conv3_3 = conv_op(conv3_2 , name='conv3_3', kh= 3, kw= 3, n_out= 256 , dh= 1, dw= 1 , p=p)\n",
    "\n",
    "    pool3 = mpool_op(conv3_3, name= 'pool3' , kh= 2, kw= 2, dh= 2, dw=2)\n",
    "\n",
    "    conv4_1 = conv_op(pool3, name='conv4_1', kh=3, kw=3, n_out=512, dh=1, dw=1, p=p)\n",
    "    conv4_2 = conv_op(conv4_1, name='conv4_2', kh=3, kw=3, n_out=512, dh=1, dw=1, p=p)\n",
    "    conv4_3 = conv_op(conv4_2, name='conv4_3', kh=3, kw=3, n_out=512, dh=1, dw=1, p=p)\n",
    "\n",
    "    pool4 = mpool_op(conv4_3, name='pool4', kh=2, kw=2, dh=2, dw=2)\n",
    "\n",
    "    conv5_1 = conv_op(pool4, name='conv5_1', kh=3, kw=3, n_out=512, dh=1, dw=1, p=p)\n",
    "    conv5_2 = conv_op(conv5_1, name='conv5_2', kh=3, kw=3, n_out=512, dh=1, dw=1, p=p)\n",
    "    conv5_3 = conv_op(conv5_2, name='conv5_3', kh=3, kw=3, n_out=512, dh=1, dw=1, p=p)\n",
    "\n",
    "    pool5 = mpool_op(conv5_3, name='pool5', kh=2, kw=2, dh=2, dw=2)\n",
    "\n",
    "    shp = pool5.get_shape()\n",
    "    flatten_shape = shp[1].value * shp[2].value * shp[3].value\n",
    "    resh1 = tf.reshape(pool5 , [-1, flatten_shape] ,name = 'resh1')\n",
    "\n",
    "    fc6 = fc_op(resh1 , name= 'fc6', n_out= 4086, p = p)\n",
    "    fc6_drop = tf.nn.dropout(fc6 , keep_prob, name = 'fc6_drop')\n",
    "    fc7 = fc_op(fc6_drop , name= 'fc7' , n_out= 4096, p=p)\n",
    "    fc7_drop = tf.nn.dropout(fc7 , keep_prob, name = 'fc7_drop')\n",
    "    fc8 = fc_op(fc7_drop , name = 'fc8' , n_out= 1000 ,p=p)\n",
    "    softmax = tf.nn.softmax(fc8)\n",
    "    predictions = tf.argmax(softmax,1)\n",
    "    return predictions, softmax , fc8 , p\n",
    "num_batches = 100\n",
    "batch_size = 32\n",
    "def time_tensorflow_run(session, target, feed , info_string):\n",
    "    num_steps_burn_in = 10\n",
    "    total_duration = 0.0\n",
    "    total_duration_squared = 0.0\n",
    "    for i in range(num_batches * num_steps_burn_in):\n",
    "        start_time = time.time()\n",
    "        _ = session.run(target , feed_dict = feed)\n",
    "        duration = time.time() - start_time\n",
    "        if i >= num_steps_burn_in:\n",
    "            if not i % 10:\n",
    "                print('%s:step %d ,duration = %.3f'%(datetime.now(),i - num_steps_burn_in,duration))\n",
    "            total_duration += duration\n",
    "            total_duration_squared +=duration * duration\n",
    "\n",
    "    mn = total_duration /num_batches\n",
    "    vr = total_duration_squared / num_batches - mn * mn\n",
    "    sd = math.sqrt(vr)\n",
    "    print('%s: %s across $d steps , %.3f +/- %.3f sec /batch'%(datetime.now(), info_string, num_batches, mn , sd))\n",
    "\n",
    "def run_benchmark():\n",
    "    with tf.Graph().as_default():\n",
    "        image_size = 224\n",
    "        images = tf.Variable(tf.random_normal([batch_size,\n",
    "                                               image_size,\n",
    "                                               image_size,3],\n",
    "                                              dtype= tf.float32,\n",
    "                                              stddev= 0.1))\n",
    "        keep_prob = tf.placeholder(tf.float32)\n",
    "        predictions, softmax, fc8 ,p = inference_op_4(images, keep_prob)\n",
    "        writer = tf.summary.FileWriter('/home/a503wangsiqi/tensorboard/vggnet_log',tf.get_default_graph())\n",
    "        writer.close()\n",
    "        init = tf.global_variables_initializer()\n",
    "        sess = tf.Session()\n",
    "        sess.run(init)\n",
    "        #time_tensorflow_run(sess, predictions, {keep_prob:1.0}, 'Forward')\n",
    "        objective = tf.nn.l2_loss(fc8)\n",
    "        grad = tf.gradients(objective, p)\n",
    "        #time_tensorflow_run(sess, grad, {keep_prob: 0.5}, 'Forward_backward')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "run_benchmark()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
